#!/usr/bin/python
import os
import sys
import threading
from jobutil import *

__author__="qingkaishi"
__date__ ="$Apr 21, 2014 9:45:11 AM$"

alljobs = []
alljobslock = threading.Lock()

def parse_job_files(group):
    jobs = []
    for item in group:
        if item[0].endswith('.json'):
            job = ApsaraJob.parse(item[0], item[1])
        else:
            job = ApsaraJob.parse(item[1], item[0])
        jobs.append(job)
        
    alljobslock.acquire()
    for item in jobs:
        alljobs.append(item)
    alljobslock.release()

#################################################################

def cpu_count():
    ''' Returns the number of CPUs in the system
    '''
    num = 2
    if sys.platform == 'win32':
        # fetch the cpu count for windows systems
        try:
            num = int(os.environ['NUMBER_OF_PROCESSORS'])
        except (ValueError, KeyError):
            pass
    elif sys.platform == 'darwin':
        # fetch teh cpu count for MacOS X systems
        try:
            num = int(os.popen('sysctl -n hw.ncpu').read())
        except ValueError:
            pass
    else:
        # an finally fetch the cpu count for Unix-like systems
        try:
            num = os.sysconf('SC_NPROCESSORS_ONLN')
        except (ValueError, OSError, AttributeError):
            pass
 
    return num

#################################################################

alljobfiles = []

def get_all_job_files(arg, dirname, names):
    jobfiles = []
    for filePath in names:
        file = os.path.join(dirname, filePath);
        if(os.path.isfile(file)):
            if(file.endswith(".json")):
                jobfiles.append(file)
            if(file.endswith(".jobstatus")):
                jobfiles.append(file)
                
    if len(jobfiles) == 2:
        alljobfiles.append(jobfiles);

if __name__ == "__main__":
    if os.path.exists(sys.argv[1]) is True:
        os.path.walk(sys.argv[1], get_all_job_files, ())
        
    group = []
    allgroups = []
    for item in alljobfiles:
        group.append(item)
        if len(group) == 100:
            allgroups.append(group)
    allgroups.append(group)
    
    cpus = cpu_count()
    for ag in allgroups:
        threadlist = []
        tempthread = threading.Thread(target=parse_job_files, args=(ag, ))
        threadlist.append(tempthread)
        
        if len(threadlist) == cpus:
            for at in threadlist:
                at.start()
            for at in threadlist:
                at.join();
                
    for at in threadlist:
        at.start()
    for at in threadlist:
        at.join();
        